/*

 Program comparedensities.do estimates densities, draws density graphs
 and calculates excess mass measures, separately by demographic cells.

 Kumler, Verhoogen, Frias "Enlisting Employees ..." REStat forthcoming

*/


***************** housekeeping ******************;

#delimit;
set more off;

do ${code}housekeeping.do;

tempfile d1;


*******************************************************************************;
********* excess mass, by metro area and age **********************;
*******************************************************************************;

* parameters;
local fyr = 1988;
local lyr = 2003;
local fqtr = 2;
local lqtr = 2;
local fage_cat2 = 1;
local lage_cat2 = 5;


  local bylist1 =  "year male age_cat2 metro_area";
  local bylist2 =  "year qtr male age_cat2 metro_area";
  local eneu_filename = "eneu_pctiles_pqy";
  local filename = "excmasspqy_metro";


********** get ENEU data ready **********;

 clear;
 use ${work}eneu_forimss_19882003;

 collapse (p10) rsal7_eneu_p10 = rsal7 (p15) rsal7_eneu_p15 = rsal7 (p20) rsal7_eneu_p20 = rsal7
     (p25) rsal7_eneu_p25 = rsal7 (p30) rsal7_eneu_p30 = rsal7 (p33) rsal7_eneu_p33 = rsal7
    (p35) rsal7_eneu_p35 = rsal7 (p40) rsal7_eneu_p40 = rsal7 (p45) rsal7_eneu_p45 = rsal7
    (p50) rsal7_eneu_p50 = rsal7  (p55) rsal7_eneu_p55 = rsal7
    (p60) rsal7_eneu_p60 = rsal7 (p65) rsal7_eneu_p65 = rsal7
    (p67) rsal7_eneu_p67 = rsal7 (p70) rsal7_eneu_p70 = rsal7
    (p75) rsal7_eneu_p75 = rsal7
    (p80) rsal7_eneu_p80 = rsal7 (p90)
    rsal7_eneu_p90 = rsal7 [pw = factor], by(`bylist1');

 save ${tmp}`eneu_filename', replace;

*** Merge ENEU pctiles into IMSS data, recover corresponding IMSS pctiles ***;

 forval yr=`fyr'/`lyr'
 {;
  forval qtr=`fqtr'/`lqtr'
  {;

   local month=`qtr'*3;
   di "year = `yr', quarter = `qtr', month = `month'";
   clear;
   use ${work}imss_forhh_`yr'`month';

   forval age_cat2 = `fage_cat2'/`lage_cat2'
   {;
    forval male = 0/1
    {;

     foreach metro_area of numlist 1/12 20/23
     {;

       di "year=`yr', quarter=`qtr', month=`month', age_cat2=`age_cat2', male=`male', metro_area=`metro_area' " ;

      preserve;

      keep if age_cat2 == `age_cat2' & male == `male' & metro_area == `metro_area';

      merge m:1 `bylist1' using ${tmp}`eneu_filename', keep(3);
      tab _merge;
      drop _merge;

      if _N>0
      {;
       sort rsal7net;
       gen imss_quantile = _n/_N;

       foreach pctile in 10 15 20 25 30 33 35 40 45 50 55 60 65 67 70 75 90
       {;
        gen excmass_p`pctile' = imss_quantile - (`pctile'/100) if rsal7_eneu_p`pctile'>=rsal7net &
           rsal7_eneu_p`pctile'<rsal7net[_n+1];
       };

       collapse (max) excmass*, by(year qtr age_cat2 male metro_area);
      };
      else if _N==0
      {;
       gen excmass_p25=.;
     };

      save ${tmp}`filename'_`yr'_`qtr'_`age_cat2'_`male'_`metro_area', replace;
      restore;
     };
    };
   };
  };
 };


******** stack estimates of excess mass **************;

 clear;
 set obs 0;
 gen male=.;

 forval yr=`fyr'/`lyr'
 {;
  forval qtr=`fqtr'/`lqtr'
  {;
   forval age_cat2 = `fage_cat2'/`lage_cat2'
   {;
    forval male = 0/1
    {;
     foreach metro_area of numlist 1/12 20/23
      {;
       append using ${tmp}`filename'_`yr'_`qtr'_`age_cat2'_`male'_`metro_area';
     };
   };
  };
 };
};

order `bylist2';
save ${work}`filename', replace;



*******************************************************************************;
****** excess mass, by metro area, firm size, age and sector ******************;
*******************************************************************************;

* parameters;
local fyr = 1988;
local fyr = 1989; * EV: UNTIL CODE RE-RUNS!;
local lyr = 2003;
local fqtr = 2;
local lqtr = 2;
local fage_cat2 = 1;
local lage_cat2 = 5;

  local bylist1 =  "year male firmsize3 metro_area age_cat2 sector";
  local bylist2 =  "year qtr male firmsize3 metro_area age_cat2 sector";
  local eneu_filename = "eneu_pctiles_pqyfsmetroagesec";
  local filename = "excmasspqyfs_metroagesec";



 ********** get ENEU data ready **********;

 clear;
 use ${work}eneu_forimss_19882003;

 collapse (p10) rsal7_eneu_p10 = rsal7 (p15) rsal7_eneu_p15 = rsal7 (p20) rsal7_eneu_p20 = rsal7
     (p25) rsal7_eneu_p25 = rsal7 (p30) rsal7_eneu_p30 = rsal7 (p33) rsal7_eneu_p33 = rsal7
    (p35) rsal7_eneu_p35 = rsal7 (p40) rsal7_eneu_p40 = rsal7 (p45) rsal7_eneu_p45 = rsal7
    (p50) rsal7_eneu_p50 = rsal7  (p55) rsal7_eneu_p55 = rsal7
    (p60) rsal7_eneu_p60 = rsal7 (p65) rsal7_eneu_p65 = rsal7
    (p67) rsal7_eneu_p67 = rsal7 (p70) rsal7_eneu_p70 = rsal7
    (p75) rsal7_eneu_p75 = rsal7
    (p80) rsal7_eneu_p80 = rsal7 (p90)
    rsal7_eneu_p90 = rsal7 [pw = factor], by(`bylist1');

 save ${tmp}`eneu_filename', replace;

 *** Merge ENEU pctiles into IMSS data, recover corresponding IMSS pctiles ***;

 forval yr=`fyr'/`lyr'
 {;
  forval qtr=`fqtr'/`lqtr'
  {;

   local month=`qtr'*3;
   di "year = `yr', quarter = `qtr', month = `month'";
   clear;
   use ${work}imss_forhh_`yr'`month';

   forval firmsize3 = 1/5
   {;
    forval male = 0/1
    {;

     foreach metro_area of numlist 1/12 20/23
     {;
      forvalues age_cat2 = 1/5
      {;
       forvalues sector = 1/3
       {;

        di "year=`yr', quarter=`qtr', firmsize3=`firmsize3', male=`male', metro_area=`metro_area',
            age_cat2 = `age_cat2', sector = `sector' " ;

        preserve;

        keep if firmsize3 == `firmsize3' & male == `male' & metro_area == `metro_area' &
            age_cat2==`age_cat2' & sector == `sector';

        merge m:1 `bylist1' using ${tmp}`eneu_filename', keep(3);
        drop _merge;

        * need to allow for the possibility that there are no ENEU obs and hence no obs left after merge;
        if _N>0
        {;
         sort rsal7net;
         gen imss_quantile = _n/_N;

         foreach pctile in 10 15 20 25 30 33 35 40 45 50 55 60 65 67 70 75 90
         {;
          gen excmass_p`pctile' = imss_quantile - (`pctile'/100) if rsal7_eneu_p`pctile'>=rsal7net &
              rsal7_eneu_p`pctile'<rsal7net[_n+1];
         };

         collapse (max) excmass*, by(`bylist2');
         };

       save ${tmp}`filename'_`yr'_`qtr'_`firmsize3'_`male'_`metro_area'_`age_cat2'_`sector', replace;
       restore;
       };
      };
     };
    };
   };
  };
 };


 ******** stack estimates of excess mass **************;

 clear;
 set obs 0;
 gen male=.;

 forval yr=`fyr'/`lyr'
 {;
  forval qtr=`fqtr'/`lqtr'
  {;
   forval firmsize3 = 1/5
   {;
    forval male = 0/1
    {;
     foreach metro_area of numlist 1/12 20/23
     {;
      forvalues age_cat2=1/5
      {;
       forvalues sector=1/3
       {;

        append using ${tmp}`filename'_`yr'_`qtr'_`firmsize3'_`male'_`metro_area'_`age_cat2'_`sector';

        * the following is to deal with cases with no observations for which excmass was not calculated;
        keep excmass* `bylist2';

       };
      };
     };
    };
   };
  };
 };

 order `bylist2';
 save ${work}`filename', replace;


log close;











